import pandas as pd
import requests
import urllib.request, urllib.error
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
import time
import logging
from tqdm import tqdm
from retry import retry

# constant
WOWRANK = {
    'CN': 0,
    '副会长': 1,
    '团长sama': 2,
    '黎明之光': 3,
    '远近闻名': 4,
    '声名鹊起': 5,
    '登堂入室': 6,
    '成名在望': 7,
    '崭露头角': 8,
    '初来乍到': 9,
}
lk_10raid = 14922
lk_25raid = 14923
icc_10raid = 15041
icc_25raid = 15042
fall_of_lk_category = 15062
BATCH = 20
LOG_BATCH = 50
ACHIEVEMENT_TPE = ThreadPoolExecutor(max_workers=BATCH)
SUMMARY_TPE = ThreadPoolExecutor(max_workers=BATCH)
STATISTICS_TPE = ThreadPoolExecutor(max_workers=BATCH)

h8_10_need = {
    "Lord Marrowgar kills (Heroic Icecrown 10 player)": 0,
    "Gunship Battle victories (Heroic Icecrown 10 player)": 1,
    "Deathbringer kills (Heroic Icecrown 10 player)": 2,
    "Festergut kills (Heroic Icecrown 10 player)": 3,
    "Rotface kills (Heroic Icecrown 10 player)": 4,
    "Blood Prince Council kills (Heroic Icecrown 10 player)": 5,
    "Valithria Dreamwalker rescues (Heroic Icecrown 10 player)": 6,
    "Blood Queen Lana'thel kills (Heroic Icecrown 10 player)": 7,
    "Sindragosa kills (Heroic Icecrown 10 player)": 8,
    "Professor Putricide kills (Heroic Icecrown 10 player)": 9,
}
h8_25_need = {
    "Lord Marrowgar kills (Heroic Icecrown 25 player)": 0,
    "Gunship Battle victories (Heroic Icecrown 25 player)": 1,
    "Deathbringer kills (Heroic Icecrown 25 player)": 2,
    "Festergut kills (Heroic Icecrown 25 player)": 3,
    "Rotface kills (Heroic Icecrown 25 player)": 4,
    "Blood Prince Council kills (Heroic Icecrown 25 player)": 5,
    "Valithria Dreamwalker rescues (Heroic Icecrown 25 player)": 6,
    "Blood Queen Lana'thel kills (Heroic Icecrown 25 player)": 7,
    "Sindragosa kills (Heroic Icecrown 25 player)": 8,
    "Professor Putricide kills (Heroic Icecrown 25 player)": 9,
}

def safe2int(s):
    try:
        tries = int(s)
        return 1 if tries > 0 else 0
    except ValueError:
        return 0

@retry(tries=3)
def do_request(url, method, **kwargs):
    head = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36' }
    try:
        if method.lower() == 'get':
            response = requests.get(url, headers=head)
        elif method.lower() == 'post':
            response = requests.post(url, {'category': kwargs['category']}, headers=head)
        return response
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
        return "error"

def get_soup(url, category):
    response = do_request(url, 'post', category=category)
    doc = response.json()['content']
    if doc == 'error':
        raise RuntimeError("error response from url!")
    return BeautifulSoup(doc, features="lxml")


def get_summary(name):
    """人物summary页面可以拿到专精pvp专业二专这些信息"""
    url = 'https://armory.warmane.com/character/{}/Lordaeron/summary'.format(name)
    response = do_request(url, 'get')
    doc = response.content
    if doc == 'error':
        raise RuntimeError("error response from url!")

    soup = BeautifulSoup(doc, features='lxml')
    sepcs = soup.find('div', attrs={'class':'specialization'})

    sepcs = sepcs.find_all('div', {'class': 'text'})
    sepc_res = ["", "", "", ""]
    idx = 0
    for sepc in sepcs:
        sepc_res[idx] = sepc.contents[0].replace("\\n", "").strip()
        idx += 1
        sepc_res[idx] = sepc.contents[1].text
        idx += 1

    profs = soup.find('div', attrs={'class':'profskills'})
    prof_res = ["", "", "", "", "0"] # 前四个位置是2个专业x点数,第五位专门看工程
    # 有的人可能没学专业
    if profs:
        profs = profs.find_all('div', {'class': 'text'})
        idx = 0
        for prof in profs:
            prof_name = prof.contents[0].replace("\\n", "").strip()
            prof_point = prof.contents[1].text.split(' / ')[0]
            prof_res[idx] = prof_name
            idx += 1
            prof_res[idx] = prof_point
            idx += 1
            if prof_name == 'Engineering':
                prof_res[4] = prof_point
    return sepc_res + prof_res


def safe_get_date(tag):
    if tag:
        return tag.text.replace('Earned ', '')
    else:
        return '-'


def get_statistics(name):
    url = 'https://armory.warmane.com/character/{}/Lordaeron/statistics'.format(name)
    soup = get_soup(url, fall_of_lk_category)

    kill_table = soup.find('tbody', attrs={'id': 'data-table-list'})
    h8_10_check = [0] * 10
    h8_25_check = [0] * 10
   
    for tr in kill_table.find_all('tr'):
        parsed = tr.find_all('td')
        if parsed[0].contents[0] in h8_10_need:
            h8_10_check[h8_10_need[parsed[0].contents[0]]] = safe2int(parsed[1].contents[0])
        if parsed[0].contents[0] in h8_25_need:
            h8_25_check[h8_25_need[parsed[0].contents[0]]] = safe2int(parsed[1].contents[0])
    return {
        '8h_10': sum(h8_10_check[:8]) == 8,
        '8h_25': sum(h8_25_check[:8]) == 8,
        '9h_10': sum(h8_10_check[:9]) == 9,
        '9h_25': sum(h8_25_check[:9]) == 9,
        '10h_10': sum(h8_10_check[:10]) == 10,
        '10h_25': sum(h8_25_check[:10]) == 10,
    }


def get_achievement(name):
    url = 'https://armory.warmane.com/character/{}/Lordaeron/achievements'.format(name)

    all_task, result = [], []
    categories = [lk_10raid, lk_25raid, icc_10raid, icc_25raid]
    
    with ThreadPoolExecutor(max_workers=4) as pool:
        for category in categories:
            all_task.append(pool.submit(get_soup, *(url, category)))    
        result = [i.result() for i in all_task]

    if len(result) != len(categories):
        raise RuntimeError("length is not identity with categories")

    soup_html_lk_10raid = result[0]
    soup_html_lk_25raid = result[1]
    soup_html_icc_10 = result[2]
    soup_html_icc_25 = result[3]
    
    # 巫妖王-10人团队
    rs_hc10 = safe_get_date(soup_html_lk_10raid.find('div', attrs={'class': 'achievement', 'id': 'ach4818'}).find('div', attrs={'class': 'date'}))
    # 巫妖王-25人团队
    rs_hc25 = safe_get_date(soup_html_lk_25raid.find('div', attrs={'class': 'achievement', 'id': 'ach4816'}).find('div', attrs={'class': 'date'}))
    # 巫妖王的陨落-10人
    icc_10hc_lk = safe_get_date(soup_html_icc_10.find('div', attrs={'class': 'achievement', 'id': 'ach4583'}).find('div', attrs={'class': 'date'}))
    icc_10nm_all = safe_get_date(soup_html_icc_10.find('div', attrs={'class': 'achievement', 'id': 'ach4532'}).find('div', attrs={'class': 'date'}))
    icc_10hc_all = safe_get_date(soup_html_icc_10.find('div', attrs={'class': 'achievement', 'id': 'ach4636'}).find('div', attrs={'class': 'date'}))
    # 巫妖王的陨落-25人
    icc_25hc_lk = safe_get_date(soup_html_icc_25.find('div', attrs={'class': 'achievement', 'id': 'ach4584'}).find('div', attrs={'class': 'date'}))
    icc_25nm_all = safe_get_date(soup_html_icc_25.find('div', attrs={'class': 'achievement', 'id': 'ach4608'}).find('div', attrs={'class': 'date'}))
    icc_25hc_all = safe_get_date(soup_html_icc_25.find('div', attrs={'class': 'achievement', 'id': 'ach4637'}).find('div', attrs={'class': 'date'}))
    return {
        'rs_hc10': rs_hc10,
        'icc_10hc_lk': icc_10hc_lk,
        'icc_10nm_all': icc_10nm_all,
        'icc_10hc_all': icc_10hc_all,
        'rs_hc25': rs_hc25,
        'icc_25hc_lk': icc_25hc_lk,
        'icc_25nm_all': icc_25nm_all,
        'icc_25hc_all': icc_25hc_all,
    }


if __name__ == "__main__":

    start_time = int(time.time())

    df = pd.read_html('CN-DKP.html',encoding='utf-8')

    ndf = pd.DataFrame()
    ndf = df[0]

    ndf.insert(1,'Main',[""] * len(ndf))
    ndf.insert(4,'Order',[""] * len(ndf))

    for i in range(0, len(ndf)):
        ndf.iloc[i,4] = WOWRANK.get(ndf.iloc[i,3], len(WOWRANK)) 

        if len(str.split(ndf.iloc[i,0])) == 2:
            [ndf.iloc[i,0],ndf.iloc[i,1]] = str.split(ndf.iloc[i,0])
            ndf.iloc[i,1] = ndf.iloc[i,1].replace("(","")
            ndf.iloc[i,1] = ndf.iloc[i,1].replace(")","")
        else:
            ndf.iloc[i,1]=ndf.iloc[i,0]

    nndf = ndf.sort_values(by=['Order','Main'], ascending=[True,True])
    nndf = nndf.reset_index(drop=True)
    nndf = nndf.drop('Order', axis=1)
    # 测试用
    # nndf = nndf[: 100]
    total_lines = len(nndf)

    achievement_tasks, achievement_result = [], []
    summary_tasks, summary_result = [], []
    kill_tasks, kill_result = [], []
    update_times = []

    for i in range(0, total_lines):
        if i % LOG_BATCH == 0:
            print(f"commit {i}th-{min(i+LOG_BATCH-1, total_lines)}th requests, total {total_lines}...")
        id = nndf.loc[i, 'Name']
        achievement_tasks.append(ACHIEVEMENT_TPE.submit(get_achievement, id))
        summary_tasks.append(SUMMARY_TPE.submit(get_summary, id))
        kill_tasks.append(STATISTICS_TPE.submit(get_statistics, id))

    for i in tqdm(range(0, len(achievement_tasks))):
        achievement_result.append(achievement_tasks[i].result())
        summary_result.append(summary_tasks[i].result())
        kill_result.append(kill_tasks[i].result())
        current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        update_times.append(current_time)
        if i % LOG_BATCH == 0:
            print(f"\nsucceed to retrieve results of {i}th-{min(i+LOG_BATCH-1, total_lines)}th, total {total_lines}...")
            time.sleep(1)

    for i in range(0, total_lines):
        id = nndf.loc[i, 'Name']
        try:
            this_char_achievement = achievement_result[i]
            this_char_summary = summary_result[i]
            this_char_kill = kill_result[i]

            nndf.loc[i, "专精1"] = this_char_summary[0]
            nndf.loc[i, "天赋1"] = this_char_summary[1]
            nndf.loc[i, "专精2"] = this_char_summary[2]
            nndf.loc[i, "天赋2"] = this_char_summary[3]
            nndf.loc[i, "专业1"] = this_char_summary[4]
            nndf.loc[i, "点数1"] = this_char_summary[5]
            nndf.loc[i, "专业2"] = this_char_summary[6]
            nndf.loc[i, "点数2"] = this_char_summary[7]
            nndf.loc[i, "工程"] = this_char_summary[8]

            nndf.loc[i, "10_HC_RS"] = this_char_achievement['rs_hc10']
            nndf.loc[i, "10_HC_ICC_LK"] = this_char_achievement['icc_10hc_lk']
            nndf.loc[i, "10_NM_ICC_全通"] = this_char_achievement['icc_10nm_all']
            nndf.loc[i, "10_HC_ICC_全通"] = this_char_achievement['icc_10hc_all']
            nndf.loc[i, "25_HC_RS"] = this_char_achievement['rs_hc25']
            nndf.loc[i, "25_HC_ICC_LK"] = this_char_achievement['icc_25hc_lk']
            nndf.loc[i, "25_NM_ICC_全通"] = this_char_achievement['icc_25nm_all']
            nndf.loc[i, "25_HC_ICC_全通"] = this_char_achievement['icc_25hc_all']
    
            nndf.loc[i, "10_8HC"] = this_char_kill['8h_10']
            nndf.loc[i, "25_8HC"] = this_char_kill['8h_25']
            nndf.loc[i, "10_9HC"] = this_char_kill['9h_10']
            nndf.loc[i, "25_9HC"] = this_char_kill['9h_25']
            nndf.loc[i, "10_10HC"] = this_char_kill['10h_10']
            nndf.loc[i, "25_10HC"] = this_char_kill['10h_25']
    
        except Exception as e:
            print(f"error in process id {id}")
            logging.exception(e)
            
    nndf["更新时间"] = update_times
    xlsxname = 'CN-DKP-' + time.strftime('%Y-%m-%d %H%M%S', time.localtime()) + '.xlsx'
    nndf.to_excel(xlsxname, sheet_name='Sheet1', index=True) # index false为不写入索引

    ACHIEVEMENT_TPE.shutdown()
    SUMMARY_TPE.shutdown()
    STATISTICS_TPE.shutdown()

    print(f"抓取{total_lines}条信息结束，用时{int(time.time() - start_time)}秒")

